﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Data.SqlClient;
using System.Collections;
using System.Xml;

namespace IKE_crawler
{
    public partial class Form1 : Form
    {
        Dictionary<string, int> pageNameAndID = new Dictionary<string, int>();
        Dictionary<string, int> wordId = new Dictionary<string, int>();
        Dictionary<int, Dictionary<int, int>> globalLinks = new Dictionary<int, Dictionary<int, int>>(); //<idstranky,<na kterou stranku (id), pocet odkazu na ni>>
        Int64 totalWordCount;
        int g_pageLargestID;
        enum TagType
        {
            title,                  //titulek stranky
            link,                   // [[ ]]
            header1,                // === ===
            header2,                // == ==
            highlight1,             // ''' '''
            highlight2,             // '' ''    
            normal                  //jakykoliv bezny text
        }

        private Dictionary<TagType, int> TagWeight = new Dictionary<TagType, int>();

        public Form1()
        {
            InitializeComponent();
        }


        private void Form1_Load(object sender, EventArgs e)
        {
            TagWeight.Add(TagType.title, 240);
            TagWeight.Add(TagType.link, 5);
            TagWeight.Add(TagType.header1, 80);
            TagWeight.Add(TagType.header2, 40);
            TagWeight.Add(TagType.highlight1, 10);
            TagWeight.Add(TagType.highlight2, 5);
            TagWeight.Add(TagType.normal, 1);
        }

        private void btnBrowse_Click(object sender, EventArgs e)
        {
            if (openFileDialog1.ShowDialog() == DialogResult.OK)
            {
                txbPath.Text = openFileDialog1.FileName;
            }
        }

        //provede predzpracovani stranek, vybuduje index stranek s jajich jmeny a id
        private void btnPrepare_Click(object sender, EventArgs e)
        {
            btnPrepare.Enabled = false;
            txbPath.Enabled = false;
            btnBrowse.Enabled = false;
            btnGenerateLinks.Enabled = false;
            btnGenerateStopWords.Enabled = false;

            g_pageLargestID = 0;
            StreamReader sr = new StreamReader(txbPath.Text);
            string line;
            bool inPage = false;
            int pageID = -1;
            ArrayList onePage = new ArrayList();
            DateTime start = DateTime.Now;
            int counter = 0;
            string pageName = "";
            richTextBox1.Text = "Zahajena priprava souboru: " + DateTime.Now.ToString() + "\n";

            bool addedToList = false;

            if (!Directory.Exists("InputFiles"))
            {
                Directory.CreateDirectory("InputFiles");
            }

            int pagesInMemory = 0;

            while ((line = sr.ReadLine()) != null)
            {
                line = line.Trim();
                if (line == "<page>")
                {
                    counter++;
                    inPage = true;
                    pageID = -1;
                    pageName = "";
                    addedToList = false;
                    if (pagesInMemory == 20)
                    {
                        onePage = new ArrayList();
                        pagesInMemory = 1;
                    }
                    else
                    {
                        pagesInMemory++;
                    }

                    if (counter % 200 == 0)
                    {
                        Application.DoEvents();
                        lblStatus.Text = "Zpracovavam stranku " + counter;
                    }
                }
                else if (line == "</page>")
                {
                    inPage = false;
                    onePage.Add("</page>");
                    if (chbGenerateInputFiles.Checked)
                    {
                        if (pagesInMemory == 20)
                        {


                            StreamWriter sw = new StreamWriter("InputFiles/Page" + pageID + ".xml");
                            sw.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
                            sw.WriteLine("<pages>");
                            foreach (object newLine in onePage)
                            {
                                sw.WriteLine(newLine.ToString());
                            }
                            sw.WriteLine("</pages>");
                            sw.Close();
                        }
                    }
                }

                if (inPage)
                {
                    onePage.Add(line);
                    if (pageID == -1)
                    {
                        Regex tag = new Regex("<([a-zA-Z0-9/]+)([^>]*)>([^<]*)");
                        MatchCollection results = tag.Matches(line);
                        Match result = results[0];
                        string tagName = result.Groups[1].Value.Trim();
                        if (tagName == "id")
                        {
                            pageID = int.Parse(result.Groups[3].Value.Trim());
                            if (pageID > g_pageLargestID)
                            {
                                g_pageLargestID = pageID;
                            }
                        }
                    }
                    if (pageName == "")
                    {
                        Regex tag = new Regex("<([a-zA-Z0-9/]+)([^>]*)>([^<]*)");
                        MatchCollection results = tag.Matches(line);
                        Match result = results[0];
                        string tagName = result.Groups[1].Value.Trim();
                        if (tagName == "title")
                        {
                            pageName = result.Groups[3].Value.Trim();
                        }
                    }

                    if ((pageName != "") && (pageID != -1) && (!addedToList))
                    {
                        pageNameAndID.Add(pageName, pageID);
                        addedToList = true;
                    }
                }

            }
            sr.Close();

            if (!Directory.Exists("OutputFiles"))
            {
                Directory.CreateDirectory("OutputFiles");
            }
            XmlTextWriter writer = new XmlTextWriter("OutputFiles/Pages.xml", System.Text.Encoding.UTF8);
            writer.Formatting = Formatting.Indented;
            writer.Indentation = 3;


            writer.WriteStartDocument();
            writer.WriteStartElement("Pages");

            foreach (KeyValuePair<string, int> pair in pageNameAndID)
            {
                writer.WriteStartElement("Page");
                writer.WriteElementString("name", pair.Key);
                writer.WriteElementString("id", pair.Value.ToString());
                writer.WriteEndElement();
            }
            writer.Close();
            TimeSpan duration = DateTime.Now - start;
            richTextBox1.Text += "priprava souboru ukoncena: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani pripravy: " + duration.ToString() + "\n";
            richTextBox1.Text += "zpracovano stranek: " + counter + "\n";
            richTextBox1.Text += "==================\n";

            //btnStart.Enabled = true; //-- pro DOK odkomentovat
            //btnGenerateStopWords.Enabled = true; //-- pro DOK odkomentovat
            btnGenerateLinks.Enabled = true; //cast pro GAL
        }

        //Spusti kompletni crawlovani stranek, indexovani slov a budovani ranku, - DOK part
        private void btnStart_Click(object sender, EventArgs e)
        {
            btnStart.Enabled = false;
            btnGenerateStopWords.Enabled = false;
            Dictionary<int, Dictionary<string, WordInfo>> WordsOnPages = new Dictionary<int, Dictionary<string, WordInfo>>();//<idStranky,<slovo,wordInfo(pocetvyskytu,vaha)>>
            DateTime start = DateTime.Now;
            int pagesCount = 0;
            ArrayList pagesToSave = new ArrayList();
            int wop_count = 0;
            richTextBox1.Text += "zahajeno crawlovani: " + DateTime.Now.ToString() + "\n";
            Application.DoEvents();


            if (!Directory.Exists("OutputFiles/WordsOnPages/"))
            {
                Directory.CreateDirectory("OutputFiles/WordsOnPages/");
            }

            int rangeToSave = 1;//pokud id stranky prekroci rangeToSave*100 ulozime stranky do souboru

            ArrayList savedWordsIDs = new ArrayList();

            XmlDocument xmlDoc = new XmlDocument();
            XmlDeclaration xmlDeclaration = xmlDoc.CreateXmlDeclaration("1.0", "utf-8", null);

            XmlElement rootNode = xmlDoc.CreateElement("Words");
            xmlDoc.InsertBefore(xmlDeclaration, xmlDoc.DocumentElement);
            xmlDoc.AppendChild(rootNode);

            Dictionary<int, Dictionary<int, int>> pw_dict = new Dictionary<int, Dictionary<int, int>>(); //<id slova,<idstranky,vahaStrankyProTotoSlovo>
            Dictionary<int, int> word_count = new Dictionary<int, int>(); //<idSlova,pocet>
            Dictionary<int, string> word_names = new Dictionary<int, string>(); //<idslova,jmeno slova>

            int pagesToProcess = 8000;// g_pageLargestID; //vsechny = g_pageLargestID
            for (int i = 0; i <= pagesToProcess; i++)
            {
                if (File.Exists("InputFiles/Page" + i + ".xml"))
                {

                    pagesCount++;
                    XmlDocument doc = new XmlDocument();
                    doc.Load("InputFiles/Page" + i + ".xml");
                    XmlNodeList id = doc.SelectNodes("//pages/page/id");

                    XmlNodeList name = doc.SelectNodes("//pages/page/title");

                    XmlNodeList text = doc.SelectNodes("//pages/page/revision/text");

                    for (int k = 0; k < id.Count; k++)
                    {
                        int iid = int.Parse(id[k].InnerText);
                        string sName = name[k].InnerText;
                        string sText = text[k].InnerText;

                        if ((iid > rangeToSave * 100) || (i == pagesToProcess))//stranky 1-100,101-200,... v jednom souboru
                        {
                            wop_count++;
                            rangeToSave++;
                            XmlTextWriter writer = new XmlTextWriter("OutputFiles/WordsOnPages/wop" + wop_count + ".xml", System.Text.Encoding.UTF8);
                            writer.Formatting = Formatting.Indented;
                            writer.Indentation = 3;


                            writer.WriteStartDocument();
                            writer.WriteStartElement("Pages");

                            foreach (int j in pagesToSave)
                            {
                                writer.WriteStartElement("Page");
                                writer.WriteElementString("id", j.ToString());
                                foreach (KeyValuePair<string, WordInfo> pair in WordsOnPages[j])
                                {
                                    writer.WriteStartElement("Word");
                                    writer.WriteElementString("id", wordId[pair.Key].ToString());
                                    writer.WriteElementString("weight", pair.Value.Weight.ToString());
                                    writer.WriteEndElement();

                                    if (pw_dict.ContainsKey(wordId[pair.Key]))
                                    {
                                        pw_dict[wordId[pair.Key]].Add(j, pair.Value.Weight);
                                        word_count[wordId[pair.Key]] += pair.Value.Count;
                                    }
                                    else
                                    {
                                        pw_dict.Add(wordId[pair.Key], new Dictionary<int, int>());
                                        word_count.Add(wordId[pair.Key], pair.Value.Count);
                                        word_names.Add(wordId[pair.Key], pair.Key);
                                    }
                                }
                                writer.WriteEndElement();

                            }
                            writer.Close();
                            pagesToSave.Clear();
                            WordsOnPages.Clear();
                            GC.Collect();
                            GC.WaitForPendingFinalizers();

                        }
                        lblStatus.Text = "zpracovavam stranku id" + (i) + " z celkem " + g_pageLargestID;
                        Application.DoEvents();
                        WordsOnPages.Add(iid, crawlPage(sText, iid, sName));
                        pagesToSave.Add(iid);

                    }


                }
            }

            xmlDoc.Save("pw.xml");
            TimeSpan duration = DateTime.Now - start;
            richTextBox1.Text += "crawlovani ukonceno: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani crawlovani: " + duration.ToString() + "\n";
            richTextBox1.Text += "zpracovano stranek (priblizne): " + pagesCount * 20 + "\n";
            richTextBox1.Text += "==================\n";


            #region test
            start = DateTime.Now;
            richTextBox1.Text += "zahajeno generovani pomocnych seznamu: " + DateTime.Now.ToString() + "\n";
            Application.DoEvents();
            if (!Directory.Exists("OutputFiles/PagesWeight/"))
            {
                Directory.CreateDirectory("OutputFiles/PagesWeight/");
            }

            XmlTextWriter writer2 = new XmlTextWriter("OutputFiles/Words.xml", System.Text.Encoding.UTF8);
            writer2.Formatting = Formatting.Indented;
            writer2.Indentation = 3;


            writer2.WriteStartDocument();
            writer2.WriteStartElement("Words");

            int pw_count = 0;
            Dictionary<int, Dictionary<int, int>> pw_pom = new Dictionary<int, Dictionary<int, int>>();
            int wCount = 0;

            foreach (KeyValuePair<int, Dictionary<int, int>> pair in pw_dict) //pro kazde slovo
            {
                pw_pom.Add(pair.Key, pair.Value);

                wCount++;

                writer2.WriteStartElement("word");

                writer2.WriteStartAttribute("name"); //Attribute "Name"
                writer2.WriteString(word_names[pair.Key]); //Attribute Value 
                writer2.WriteEndAttribute();

                writer2.WriteStartAttribute("id"); //Attribute "Name"
                writer2.WriteString(pair.Key.ToString()); //Attribute Value 
                writer2.WriteEndAttribute();

                writer2.WriteStartAttribute("count"); //Attribute "Name"
                writer2.WriteString(word_count[pair.Key].ToString()); //Attribute Value 
                writer2.WriteEndAttribute();

                //writer2.WriteElementString("name", pair.Key);
                //writer2.WriteElementString("id", pair.Value.ToString());
                //writer2.WriteElementString("count", count.ToString());

                writer2.WriteEndElement();

                if (pair.Key % 50 == 0)
                {
                    lblStatus.Text = "zpracovavam slovo " + pair.Key;
                    Application.DoEvents();
                }

                if ((pw_pom.Count % 50 == 0)) //az mame slov 50, ulozime je do xmlka
                {
                    pw_count++;
                    XmlTextWriter writer3 = new XmlTextWriter("OutputFiles/PagesWeight/pw" + pw_count + ".xml", System.Text.Encoding.UTF8);
                    writer3.Formatting = Formatting.Indented;
                    writer3.Indentation = 3;


                    writer3.WriteStartDocument();
                    writer3.WriteStartElement("Words");

                    foreach (KeyValuePair<int, Dictionary<int, int>> pair3 in pw_pom)
                    {
                        writer3.WriteStartElement("word");
                        writer3.WriteElementString("id", pair3.Key.ToString());
                        foreach (KeyValuePair<int, int> pair4 in pair3.Value)
                        {
                            writer3.WriteStartElement("page");
                            writer3.WriteElementString("id", pair4.Key.ToString());
                            writer3.WriteElementString("weight", pair4.Value.ToString());
                            writer3.WriteEndElement();
                        }
                        writer3.WriteEndElement();
                    }
                    writer3.Close();
                    pw_pom.Clear();
                }

            }
            writer2.Close();
            if (pw_pom.Count != 0) // pokud nam zbyly slova, zapiseme je do posledniho souboru
            {
                pw_count++;
                XmlTextWriter writer3 = new XmlTextWriter("OutputFiles/PagesWeight/pw" + pw_count + ".xml", System.Text.Encoding.UTF8);
                writer3.Formatting = Formatting.Indented;
                writer3.Indentation = 3;


                writer3.WriteStartDocument();
                writer3.WriteStartElement("Words");

                foreach (KeyValuePair<int, Dictionary<int, int>> pair3 in pw_pom)
                {
                    writer3.WriteStartElement("word");
                    writer3.WriteElementString("id", pair3.Key.ToString());
                    foreach (KeyValuePair<int, int> pair4 in pair3.Value)
                    {
                        writer3.WriteStartElement("page");
                        writer3.WriteElementString("id", pair4.Key.ToString());
                        writer3.WriteElementString("weight", pair4.Value.ToString());
                        writer3.WriteEndElement();
                    }
                    writer3.WriteEndElement();

                    wCount++;
                }
                writer3.Close();
                pw_pom.Clear();
            }

            duration = DateTime.Now - start;
            richTextBox1.Text += "generovani pomocnych seznamu ukonceno: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani generovani pomocnych seznamu: " + duration.ToString() + "\n";
            richTextBox1.Text += "celkem zpracovano slov: " + wCount + "\n";
            richTextBox1.Text += "==================\n";
            #endregion


            lblStatus.Text = "Zpracovani dokonceno";

            btnGenerateStopWords.Enabled = true;
        }

        private void saveWords(string text, TagType type, Dictionary<string, WordInfo> gwords)
        {
            /*Dictionary<string, int> Expressions = new Dictionary<string, int>();
            Expressions.Add("([0-9]+,[0-9]+)", 1);
            Expressions.Add("([0-9]+\\.[0-9\\.]+)", 1);

            foreach (KeyValuePair<string, int> pair in Expressions)
            {
                Regex words = new Regex(pair.Key, RegexOptions.Multiline);
                MatchCollection results = words.Matches(text);

                foreach (Match result in results)
                {
                    text = Regex.Replace(text, pair.Key, "", RegexOptions.Multiline);

                    for (int j = 1; j < result.Groups.Count; j++)
                    {
                        if (!string.IsNullOrEmpty(result.Groups[j].Value))
                        {
                            string word = result.Groups[j].Value.Trim();
                            if (gwords.ContainsKey(word))
                            {
                                gwords[word].Count++;
                                gwords[word].Weight += TagWeight[type];
                            }
                            else
                            {
                                gwords.Add(word, new WordInfo(1, TagWeight[type]));
                                if (!wordId.ContainsKey(word))
                                {
                                    wordId.Add(word, wordId.Count);
                                }
                            }
                        }
                    }
                }
            }*/

            Regex wordsExp = new Regex("\\b(\\w+)\\b");
            MatchCollection resultsMatch = wordsExp.Matches(text);
            foreach (Match result in resultsMatch)
            {
                string word = result.Groups[1].Value.Trim().ToLower();
                if (gwords.ContainsKey(word))
                {
                    gwords[word].Count++;
                    gwords[word].Weight += TagWeight[type];
                }
                else
                {
                    gwords.Add(word, new WordInfo(1, TagWeight[type]));
                    if (!wordId.ContainsKey(word))
                    {
                        wordId.Add(word, wordId.Count);
                    }
                }
            }
        }

        private void crawlTagContent(TagType type, string text, Dictionary<string, WordInfo> words, int pageID)
        {
            if (type == TagType.link)
            {
                string[] linkOuter = text.Split(new string[] { ";;" }, StringSplitOptions.None);
                if (linkOuter.Length == 2)
                {
                    string[] link = linkOuter[0].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);

                    if (link.Length > 0)
                    {

                        if (pageNameAndID.ContainsKey(link[0]))
                        {
                            if (globalLinks[pageID].ContainsKey(pageNameAndID[link[0]]))
                            {
                                globalLinks[pageID][pageNameAndID[link[0]]]++;
                            }
                            else
                            {
                                globalLinks[pageID].Add(pageNameAndID[link[0]], 1);
                            }
                        }

                        if (link.Length == 2)
                        {
                            string linkText = link[1] + linkOuter[1];
                            saveWords(linkText, TagType.link, words);
                        }
                        else
                        {
                            saveWords(link[0], TagType.link, words);
                        }
                    }
                }
                else
                {
                    string[] link = text.Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);

                    if (link.Length > 0)
                    {
                        if (pageNameAndID.ContainsKey(link[0]))
                        {
                            if (globalLinks[pageID].ContainsKey(pageNameAndID[link[0]]))
                            {
                                globalLinks[pageID][pageNameAndID[link[0]]]++;
                            }
                            else
                            {
                                globalLinks[pageID].Add(pageNameAndID[link[0]], 1);
                            }
                        }

                        if (link.Length == 2)
                        {
                            saveWords(link[1], TagType.link, words);
                        }
                        else
                        {
                            saveWords(link[0], TagType.link, words);
                        }
                    }
                }
            }
            else
            {
                saveWords(text, type, words);
            }

        }

        private Dictionary<string, WordInfo> crawlPage(string text, int pageID, string pageName)
        {
            Dictionary<string, TagType> Expressions = new Dictionary<string, TagType>();
            Dictionary<string, WordInfo> Words = new Dictionary<string, WordInfo>();
            Expressions.Add("^===(.+?)===", TagType.header1);
            Expressions.Add("\\[\\[(.+?)\\]\\](\\w*)", TagType.link);
            Expressions.Add("^==(.+?)==", TagType.header2);
            Expressions.Add("'''(.+?)'''", TagType.highlight1);
            Expressions.Add("''(.+?)''", TagType.highlight2);


            string text2 = text;
            text2 = Regex.Replace(text2, "<ref[^>]*>.+?</ref>", "", RegexOptions.Singleline);// . bude brat i konce radku
            text2 = Regex.Replace(text2, "<!--.+?-->", "", RegexOptions.Singleline);
            text2 = Regex.Replace(text2, "&nbsp;", " ");
            text2 = Regex.Replace(text2, "<.+?>", "");
            text2 = Regex.Replace(text2, "{\\|.+?\\|}", "", RegexOptions.Singleline);
            text2 = Regex.Replace(text2, "{{.+?}}", "", RegexOptions.Singleline);
            text2 = Regex.Replace(text2, "[0-9]+,[0-9]+", ""); //nebudeme indexovat cisla
            text2 = Regex.Replace(text2, "[0-9]+\\.[0-9\\.]+", "");
            text2 = Regex.Replace(text2, "[0-9]+", "");

            saveWords(pageName, TagType.title, Words);

            globalLinks.Add(pageID, new Dictionary<int, int>());

            foreach (KeyValuePair<string, TagType> pair in Expressions)
            {
                Regex words = new Regex(pair.Key, RegexOptions.Multiline);
                MatchCollection results = words.Matches(text2);

                foreach (Match result in results)
                {
                    text2 = Regex.Replace(text2, pair.Key, "", RegexOptions.Multiline);
                    if (result.Groups[1].Value.Trim().Contains(':'))
                        continue;


                    if (pair.Value == TagType.link)
                    {
                        if (!string.IsNullOrEmpty(result.Groups[2].Value))
                        {

                            string tagContent = result.Groups[1].Value.Trim() + ";;" + result.Groups[2].Value.Trim();
                            crawlTagContent(pair.Value, tagContent, Words, pageID);
                        }
                        else
                        {
                            string tagContent = result.Groups[1].Value.Trim();
                            crawlTagContent(pair.Value, tagContent, Words, pageID);
                        }
                    }
                    else
                    {
                        string tagContent = result.Groups[1].Value.Trim();
                        crawlTagContent(pair.Value, tagContent, Words, pageID);
                    }


                }

            }
            saveWords(text2, TagType.normal, Words);

            return Words;
        }

        private void btnGenerateStopWords_Click(object sender, EventArgs e)
        {
            DateTime start = DateTime.Now;
            lblStatus.Text = "Zpracovavam...";
            richTextBox1.Text += "zahajeno generovani seznamu stop slov: " + DateTime.Now.ToString() + "\n";
            Application.DoEvents();
            XmlDocument doc = new XmlDocument();
            doc.Load("OutputFiles/Words.xml");
            Application.DoEvents();
            XmlNodeList words = doc.SelectNodes("//Words/word[@count>20000]");
            Application.DoEvents();
            Dictionary<int, int> wCount = new Dictionary<int, int>();//<idSlova,pocetvyskytu>
            Dictionary<int, string> wId = new Dictionary<int, string>(); //<idslova,slovo>
            foreach (XmlNode word in words)
            {
                wCount.Add(int.Parse(word.Attributes["id"].Value), int.Parse(word.Attributes["count"].Value));
                wId.Add(int.Parse(word.Attributes["id"].Value), word.Attributes["name"].Value);
            }

            Application.DoEvents();
            var items = from k in wCount.Keys
                        orderby wCount[k] descending
                        select k;
            Application.DoEvents();

            XmlTextWriter writer = new XmlTextWriter("OutputFiles/StopWords.xml", System.Text.Encoding.UTF8);
            writer.Formatting = Formatting.Indented;
            writer.Indentation = 3;


            writer.WriteStartDocument();
            writer.WriteStartElement("Words");

            foreach (int k in items)
            {
                writer.WriteStartElement("Word");
                writer.WriteElementString("name", wId[k]);
                writer.WriteElementString("id", k.ToString());
                writer.WriteEndElement();
            }
            writer.Close();

            TimeSpan duration = DateTime.Now - start;
            richTextBox1.Text += "generovani pgenerovani seznamu stop slov: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani generovani seznamu stop slov: " + duration.ToString() + "\n";
            richTextBox1.Text += "==================\n";

            lblStatus.Text = "Zpracovani dokonceno";
        }

        private void button1_Click(object sender, EventArgs e)
        {
            btnStart.Enabled = false;
            btnGenerateStopWords.Enabled = false;

            DateTime start = DateTime.Now;
            int pagesCount = 0;
            ArrayList pagesToSave = new ArrayList();
            int wop_count = 0;
            richTextBox1.Text += "zahajeno crawlovani: " + DateTime.Now.ToString() + "\n";
            Application.DoEvents();

            Dictionary<int, string> idName = new Dictionary<int, string>();

            int rangeToSave = 1;//pokud id stranky prekroci rangeToSave*100 ulozime stranky do souboru
            int pagesToProcess;

            if (chbSmallPart.Checked)
                pagesToProcess = 200;
            else
                pagesToProcess = g_pageLargestID;
            // g_pageLargestID; //vsechny = g_pageLargestID
            for (int i = 0; i <= pagesToProcess; i++)
            {
                if (File.Exists("InputFiles/Page" + i + ".xml"))
                {

                    pagesCount++;
                    XmlDocument doc = new XmlDocument();
                    doc.Load("InputFiles/Page" + i + ".xml");
                    XmlNodeList id = doc.SelectNodes("//pages/page/id");

                    XmlNodeList name = doc.SelectNodes("//pages/page/title");

                    XmlNodeList text = doc.SelectNodes("//pages/page/revision/text");

                    for (int k = 0; k < id.Count; k++)
                    {
                        int iid = int.Parse(id[k].InnerText);
                        string sName = name[k].InnerText;
                        string sText = text[k].InnerText;

                        lblStatus.Text = "zpracovavam stranku id" + (i) + " z celkem " + pagesToProcess;
                        Application.DoEvents();
                        idName.Add(iid, sName);
                        crawlPage(sText, iid, sName);

                    }


                }
            }

            TimeSpan duration = DateTime.Now - start;
            richTextBox1.Text += "crawlovani ukonceno: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani crawlovani: " + duration.ToString() + "\n";
            richTextBox1.Text += "zpracovano stranek (priblizne): " + pagesCount * 20 + "\n";
            richTextBox1.Text += "==================\n";


            #region test
            start = DateTime.Now;
            richTextBox1.Text += "zahajeno generovani pomocnych seznamu: " + DateTime.Now.ToString() + "\n";
            Application.DoEvents();

            XmlTextWriter writer2 = new XmlTextWriter("OutputFiles/Links.xml", System.Text.Encoding.UTF8);
            writer2.Formatting = Formatting.Indented;
            writer2.Indentation = 3;


            writer2.WriteStartDocument();
            writer2.WriteStartElement("Links");
            int linksCount = 0;
            foreach (KeyValuePair<int, Dictionary<int, int>> pair in globalLinks) //pro kazdy link
            {
                linksCount++;
                lblStatus.Text = "zpracovavam stranku: " + pair.Key;
                Application.DoEvents();
                writer2.WriteStartElement("PageFrom");

                writer2.WriteStartAttribute("id"); //Attribute "Name"
                writer2.WriteString(pair.Key.ToString()); //Attribute Value 
                writer2.WriteEndAttribute();

                writer2.WriteStartAttribute("name");
                writer2.WriteString(idName[pair.Key]); //Attribute Value 
                writer2.WriteEndAttribute();

                foreach (KeyValuePair<int, int> pair2 in pair.Value)
                {
                    writer2.WriteStartElement("PageTo");

                    writer2.WriteStartAttribute("id"); //Attribute "Name"
                    writer2.WriteString(pair2.Key.ToString()); //Attribute Value 
                    writer2.WriteEndAttribute();

                    writer2.WriteStartAttribute("count"); //Attribute "Name"
                    writer2.WriteString(pair2.Value.ToString()); //Attribute Value 
                    writer2.WriteEndAttribute();

                    

                    writer2.WriteEndElement();
                }
                writer2.WriteEndElement();
            }
            writer2.Close();

            duration = DateTime.Now - start;
            richTextBox1.Text += "generovani pomocnych seznamu ukonceno: " + DateTime.Now.ToString() + "\n";
            richTextBox1.Text += "trvani generovani pomocnych seznamu: " + duration.ToString() + "\n";
            richTextBox1.Text += "celkem zpracovano stranek: " + linksCount + "\n";
            richTextBox1.Text += "==================\n";
            #endregion


            lblStatus.Text = "Zpracovani dokonceno";

            btnGenerateStopWords.Enabled = true;
        }


    }
}
